import java.io.File;
import org.w3c.dom.*;

import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.DocumentBuilder;

import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.zip.*;

import java.util.*;

/**
 * The class used to parse the input files formatted in XML format
 * It can be compressed within single ZIP file
 */
public class Parser
{
	/**
	 * Only take the value of <DOCNO> for identifier and <TEXT> for the value
	 * @param in InputStream to the file
	 * @param output (Output) a list of documents to be processed
	 */
	private static void parseEntry(InputStream in, List<XMLData> output)
	{
		try
		{
			BufferedReader is = new BufferedReader(new InputStreamReader(in));		
	
			StringBuilder content = new StringBuilder("<?xml version=\"1.0\"?>\n");
			content.append("<!DOCTYPE xsl:stylesheet ");
			
			// This part is used to remove unwanted characters
			content.append("[<!ENTITY Cx1f \"\"><!ENTITY Cx13 \"\"><!ENTITY Cx11 \"\">");
			content.append("<!ENTITY UR \"\"><!ENTITY QC \"\"><!ENTITY Cx14 \"\">");
			content.append("<!ENTITY Cx12 \"\"><!ENTITY QL \"\"><!ENTITY Cx1f \"\">");
			content.append("<!ENTITY AMP \"\"><!ENTITY HT \"\"><!ENTITY LR \"\">");
			content.append("<!ENTITY QR \"\">");
			content.append("]>");
			// ---
	
			String line = null;
			content.append("<XML>");
			while ((line = is.readLine()) != null)
			{
				content.append(line + "\n");
			}
			content.append("</XML>");
	
			ByteArrayInputStream bs = new ByteArrayInputStream(content.toString().getBytes());
	
			DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance();
		    DocumentBuilder docBuilder = docBuilderFactory.newDocumentBuilder();
		    Document doc = docBuilder.parse(new org.xml.sax.InputSource(bs));
	
			NodeList listOfDocs = doc.getElementsByTagName("DOC");
			int len = listOfDocs.getLength();
			for (int i = 0; i < len; i++)
			{
				Node eachDoc = listOfDocs.item(i);
				if (eachDoc.getNodeType() == Node.ELEMENT_NODE)
				{
					Element docElement = (Element)eachDoc;
	
					// Header
					/* not used in order to save spaces
					NodeList headerList = docElement.getElementsByTagName("HEADLINE"); 
					System.out.println(headerList.getLength());
					Element headerElement = (Element)headerList.item(0);
					NodeList headerValueList = headerElement.getChildNodes();
					String header = ((Node)headerValueList.item(0)).getNodeValue().trim();
					*/
	
					// DOCNO
					Element DocNoElement = (Element)((NodeList)docElement.getElementsByTagName("DOCNO")).item(0);					
					String DocNo = ((Node)((NodeList)DocNoElement.getChildNodes()).item(0)).getNodeValue().trim();
	
					// Text
					Element textElement = (Element)((NodeList)docElement.getElementsByTagName("TEXT")).item(0);					
					String text = "";
					if (textElement != null) text = ((Node)((NodeList)textElement.getChildNodes()).item(0)).getNodeValue().trim();
	
					if (text.length() > 1000) output.add(new XMLData(DocNo, "", text)); 	// Only take those with considerable number of characters
				}	
			}
		}
		catch (Exception ex)
		{
			ex.printStackTrace();
		}
	}

	/**
	 * 
	 * @param f the file to be read
	 * @param output (Output) a list of documents to be processed
	 * @param n limit how many documents to be read from the file
	 */
	public static void Parse(File f, List<XMLData> output, int n)
	{
		try
		{	
			FileInputStream fin = new FileInputStream(f);
			// Find the format of the file based on the extension
			if (f.getName().indexOf(".gz") >= 0)
			{
				parseEntry(new GZIPInputStream(fin), output);
			}
			if (f.getName().indexOf(".zip") >= 0)
			{
				ZipInputStream zStream = new ZipInputStream(fin);
				ZipEntry entry = null;
				ZipFile zFile = new ZipFile(f);
				// If there are many files compressed within one zip file, then only take limited number of documents
				while(((entry = zStream.getNextEntry()) != null) && (output.size() < n))
				{
					parseEntry(zFile.getInputStream(entry), output);
				}
			}
			else 
			{
				parseEntry(fin, output);
			}

		} catch(Exception ex)
		{
			ex.printStackTrace();
		}
	}
}
